<!DOCTYPE html>
<html lang="en-us">
<head><head>
    <meta name="google-site-verification" content="9vIieCe-Qpd78QOmBl63rGtIVbhY6sYyuxX3j8XWBA4" />
    <meta name="baidu-site-verification" content="LRrmH41lz7" />
    <meta charset="utf-8">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="google-site-verification" content="xBT4GhYoi5qRD5tr338pgPM5OWHHIDR6mNg1a3euekI" />
    <meta name="viewport" content="width=device-width, initial-scale=1">
    
    <meta name="description" content="通过nginx&#43;keepalived实现k8s多master高可用集群">
    
    <meta name="keyword"  content=", , 马少勋的网络日志, 马少勋的博客, Mashaoxun Blog, 博客, 个人网站, 互联网, Web, 云原生, PaaS, Istio, Kubernetes, 微服务, Microservice">
    <link rel="shortcut icon" href="/blog/img/favicon.ico">

    <title>k8s多master部署-马少勋的博客 | MaShaoXun Blog</title>

    <link rel="canonical" href="/blog/blog/post/2019-03-20-k8s-master-cluster/">

    <link rel="stylesheet" href="/blog/css/iDisqus.min.css"/>
	
    
    <link rel="stylesheet" href="/blog/css/bootstrap.min.css">

    
    <link rel="stylesheet" href="/blog/css/hux-blog.min.css">

    
    <link rel="stylesheet" href="/blog/css/syntax.css">

    
    <link rel="stylesheet" href="/blog/css/zanshang.css">

    
    <link href="//cdnjs.cloudflare.com/ajax/libs/font-awesome/4.6.3/css/font-awesome.min.css" rel="stylesheet" type="text/css">
    
    
    <script src="/blog/js/jquery.min.js"></script>
    
    
    <script src="/blog/js/bootstrap.min.js"></script>
    
    
    <script src="/blog/js/hux-blog.min.js"></script>
	
    
    
    <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.13.1/styles/github.min.css">
    <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.13.1/highlight.min.js"></script>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.13.1/languages/dockerfile.min.js"></script>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.13.1/languages/vim.min.js"></script>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.13.1/languages/yaml.min.js"></script>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.13.1/languages/go.min.js"></script>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.13.1/languages/bash.min.js"></script>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.13.1/languages/shell.min.js"></script>
    <script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.13.1/languages/nginx.min.js"></script>
    <script>hljs.initHighlightingOnLoad();</script>
</head>
</head>

<nav class="navbar navbar-default navbar-custom navbar-fixed-top">
    <div class="container-fluid">
        
        <div class="navbar-header page-scroll">
            <button type="button" class="navbar-toggle">
                <span class="sr-only">Toggle navigation</span>
                <span class="icon-bar"></span>
                <span class="icon-bar"></span>
                <span class="icon-bar"></span>
            </button>
            <a class="navbar-brand" href="/">Mashaoxun Blog</a>
        </div>

        
        
        <div id="huxblog_navbar">
            <div class="navbar-collapse">
                <ul class="nav navbar-nav navbar-right">
                    <li>
                        <a href="/blog">Home</a>
                    </li>
                    
                    <li>
                        <a href="/blog/categories/django">django</a>
                    </li>
                    
                    <li>
                        <a href="/blog/categories/docker">docker</a>
                    </li>
                    
                    <li>
                        <a href="/blog/categories/k8s">k8s</a>
                    </li>
                    
                    <li>
                        <a href="/blog/categories/redis">redis</a>
                    </li>
                    
                    <li>
                        <a href="/blog/categories/swarm">swarm</a>
                    </li>
                    
                    <li>
                        <a href="/blog/categories/%E5%AD%98%E5%82%A8">存储</a>
                    </li>
                    
                    <li>
                        <a href="/blog/categories/%E7%9B%91%E6%8E%A7">监控</a>
                    </li>
                    
                    
		    
                        <li><a href="/blog/top/about/">ABOUT</a></li>
                    

                    
		    <li>
                        <a href="/blog/search">SEARCH <img src="/blog/img/search.png" height="15" style="cursor: pointer;"></a>
		    </li>
                    
                </ul>
            </div>
        </div>
        
    </div>
    
</nav>
<script>
    
    
    
    var $body   = document.body;
    var $toggle = document.querySelector('.navbar-toggle');
    var $navbar = document.querySelector('#huxblog_navbar');
    var $collapse = document.querySelector('.navbar-collapse');

    $toggle.addEventListener('click', handleMagic)
    function handleMagic(e){
        if ($navbar.className.indexOf('in') > 0) {
        
            $navbar.className = " ";
            
            setTimeout(function(){
                
                if($navbar.className.indexOf('in') < 0) {
                    $collapse.style.height = "0px"
                }
            },400)
        }else{
        
            $collapse.style.height = "auto"
            $navbar.className += " in";
        }
    }
</script>




<style type="text/css">
    header.intro-header {
        background-image: url('/blog/img/youyou.jpg')
    }
</style>
<header class="intro-header">
    <div class="container">
        <div class="row">
            <div class="col-lg-8 col-lg-offset-2 col-md-10 col-md-offset-1">
                <div class="post-heading">
                    <div class="tags">
                        
                        <a class="tag" href="/blog/tags/k8s" title="k8s">
                            k8s
                        </a>
                        
                    </div>
                    <h1>k8s多master部署</h1>
                    <h2 class="subheading"></h2>
                    <span class="meta">
			Posted by 
			
			         &#34;马少勋&#34;
			 
			on 
			Tuesday, March 12, 2019
                        
                    </span>
                </div>
            </div>
        </div>
    </div>
</header>




<article>
    <div class="container">
        <div class="row">

            
            <div class="
                col-lg-8 col-lg-offset-2
                col-md-10 col-md-offset-1
                post-container">

                
                <header>
                    <h2>TOC</h2>
                </header>
                <nav id="TableOfContents">
<ul>
<li>
<ul>
<li><a href="#流程图">流程图</a></li>
<li><a href="#主机信息">主机信息</a></li>
<li><a href="#复制配置文件到新master02节点">复制配置文件到新master02节点</a></li>
<li><a href="#nginx-keepalived高可用-针对kube-apiserver">nginx+keepalived高可用（针对kube-apiserver）</a></li>
<li><a href="#修改所有node节点的配置文件">修改所有node节点的配置文件</a></li>
<li><a href="#验证">验证</a></li>
</ul></li>
</ul>
</nav>
                
                

<h2 id="流程图">流程图</h2>

<p><img src="http://www.mashaoxun.com/img/k8s-master-cluster.png" alt="k8s-master-cluster" /></p>

<h2 id="主机信息">主机信息</h2>

<table>
<thead>
<tr>
<th>ip地址</th>
<th>节点</th>
</tr>
</thead>

<tbody>
<tr>
<td>10.211.55.7</td>
<td>k8s-master01</td>
</tr>

<tr>
<td>10.211.55.10</td>
<td>k8s-master02</td>
</tr>

<tr>
<td>10.211.55.8</td>
<td>k8s-node01</td>
</tr>

<tr>
<td>10.211.55.9</td>
<td>k8s-node02</td>
</tr>

<tr>
<td>10.211.55.11</td>
<td>nginx01+keepalived01</td>
</tr>

<tr>
<td>10.211.55.12</td>
<td>nginx02+keepalived02</td>
</tr>

<tr>
<td>10.211.55.20</td>
<td>keepalived-vip</td>
</tr>
</tbody>
</table>

<h2 id="复制配置文件到新master02节点">复制配置文件到新master02节点</h2>

<p>登录到master01节点</p>

<pre><code>复制cfg文件
[root@master bin]# scp -r /etc/kubernetes root@10.211.55.10:/etc/
复制service文件
[root@master bin]# scp /usr/lib/systemd/system/kube-*.service root@10.211.55.10:/usr/lib/systemd/system/
复制命令文件
[root@master bin]# scp /usr/local/bin/kube* root@10.211.55.10:/usr/local/bin/
</code></pre>

<p>查看是否可用</p>

<pre><code>systemctl start kube-apiserver
systemctl start kube-scheduler
systemctl start kube-controller-manager

[root@master2 kubernetes]# ps -ef| grep kube | grep -v grep -c
3
[root@master2 kubernetes]# kubectl get no
NAME          STATUS   ROLES    AGE   VERSION
10.211.55.8   Ready    &lt;none&gt;   17h   v1.13.3
10.211.55.9   Ready    &lt;none&gt;   17h   v1.13.3
</code></pre>

<p>此时master02部署完成，但是node节点的cfg文件仍然连接的是master01</p>

<h2 id="nginx-keepalived高可用-针对kube-apiserver">nginx+keepalived高可用（针对kube-apiserver）</h2>

<p>两台主机都安装nginx和keepalived<br />
yum install nginx -y<br />
yum install keepalived -y<br />
编辑nginx01配置文件，添加如下</p>

<pre><code>stream {
    log_format main &quot;$remote_addr $upstream_addr $time_local $status&quot;;
    access_log /var/log/nginx/k8s-access.log main;
    upstream k8s-apiserver {
    server 10.211.55.7:6443;
    server 10.211.55.10:6443;
}
    server {
    listen 0.0.0.0:6443; # 接收所有来源地址，这里指接收node的地址
    proxy_pass k8s-apiserver;
}
}
</code></pre>

<p>编辑nginx02配置文件，添加如下</p>

<pre><code>stream {
    log_format main &quot;$remote_addr $upstream_addr $time_local $status&quot;;
    access_log /var/log/nginx/k8s-access.log main;
    upstream k8s-apiserver {
    server 10.211.55.7:6443;
    server 10.211.55.10:6443;
}
    server {
    listen 0.0.0.0:6443; # 接收所有来源地址，这里指接收node的地址
    proxy_pass k8s-apiserver;
}
}
</code></pre>

<p>编辑keepalived01配置文件</p>

<pre><code>! Configuration File for keepalived

global_defs {
   # 接收邮件地址
   notification_email {
     acassen@firewall.loc
     failover@firewall.loc
     sysadmin@firewall.loc
   }
   # 邮件发送地址
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id NGINX_MASTER
}

vrrp_script check_nginx {
    script &quot;/data/scripts/check_nginx.sh&quot; # 双杀脚本，当nginx进程数为0，则停止keepalived
}

vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 51 # VRRP 路由 主备相同
    priority 100    # 优先级，备服务器设置 90
    advert_int 1    # 指定VRRP 心跳包通告间隔时间，默认1秒
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        10.211.55.20/24
    }
    track_script {
        check_nginx
    }
}
</code></pre>

<p>编辑keepalived02配置文件</p>

<pre><code>! Configuration File for keepalived

global_defs {
   # 接收邮件地址
   notification_email {
     acassen@firewall.loc
     failover@firewall.loc
     sysadmin@firewall.loc
   }
   # 邮件发送地址
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id NGINX_MASTER
}

vrrp_script check_nginx {
    script &quot;/data/scripts/check_nginx.sh&quot;
}

vrrp_instance VI_1 {
    state BACKUP
    interface eth0
    virtual_router_id 51 # VRRP 路由 主备相同
    priority 90    # 优先级，备服务器设置 90
    advert_int 1    # 指定VRRP 心跳包通告间隔时间，默认1秒
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        10.211.55.20/24
    }
    track_script {
        check_nginx
    }
}
</code></pre>

<p>check_nginx脚本如下,对应keepalived配置文件中的vrrp_script。作用是kill keepalived，从而漂移vip到backup节点。</p>

<pre><code>[root@nginx01 scripts]# cat check_nginx.sh
count=$(ps -ef |grep nginx |egrep -cv &quot;grep|$$&quot;)

if [ &quot;$count&quot; -eq 0 ];then
    systemctl stop keepalived
fi
</code></pre>

<p>两台主机分别启动nginx和keepalived，确保vip在nginx01上。</p>

<pre><code>systemctl start nginx
systemctl start keepalived
</code></pre>

<h2 id="修改所有node节点的配置文件">修改所有node节点的配置文件</h2>

<p>将master的ip改成nginx+keepalived生成的vip</p>

<pre><code>sed -i s/10.211.55.7/10.211.55.20/g bootstrap.kubeconfig
sed -i s/10.211.55.7/10.211.55.20/g kubelet.kubeconfig
sed -i s/10.211.55.7/10.211.55.20/g kube-proxy.kubeconfig
</code></pre>

<p>重启node组件</p>

<pre><code>systemctl restart kubelet
systemctl restart kube-proxy
</code></pre>

<h2 id="验证">验证</h2>

<p>查看nginx转发日志，当vip在nginx01上的时候，nginx接收node（55.8，55.9）发来的请求，转发给后端master（55.7，55.10）</p>

<pre><code>[root@nginx01 keepalived]# tail -fn 10 /var/log/nginx/k8s-access.log
10.211.55.8 10.211.55.7:6443 20/Mar/2019:09:26:15 -0400 200
10.211.55.8 10.211.55.10:6443 20/Mar/2019:09:26:15 -0400 200
10.211.55.8 10.211.55.7:6443 20/Mar/2019:09:26:15 -0400 200
10.211.55.8 10.211.55.10:6443 20/Mar/2019:09:26:15 -0400 200
10.211.55.8 10.211.55.7:6443 20/Mar/2019:09:26:15 -0400 200
10.211.55.9 10.211.55.10:6443 20/Mar/2019:09:26:16 -0400 200
</code></pre>

<p>此时停止nginx01，nginx02接管vip，node与vip正常通信，从而保证了master节点的高可用，查看nginx02日志</p>

<pre><code>[root@nginx02 keepalived]# tail -fn 5 /var/log/nginx/k8s-access.log
10.211.55.9 10.211.55.7:6443 20/Mar/2019:09:25:53 -0400 200
10.211.55.9 10.211.55.10:6443 20/Mar/2019:09:25:53 -0400 200
10.211.55.9 10.211.55.10:6443 20/Mar/2019:09:25:53 -0400 200
10.211.55.9 10.211.55.7:6443 20/Mar/2019:09:25:53 -0400 200
10.211.55.9 10.211.55.10:6443 20/Mar/2019:09:25:53 -0400 200

10.211.55.8 10.211.55.7:6443 20/Mar/2019:09:46:16 -0400 200
10.211.55.9 10.211.55.10:6443 20/Mar/2019:09:46:16 -0400 200
10.211.55.8 10.211.55.7:6443 20/Mar/2019:09:46:16 -0400 200
10.211.55.9 10.211.55.10:6443 20/Mar/2019:09:46:16 -0400 200
10.211.55.8 10.211.55.7:6443 20/Mar/2019:09:46:16 -0400 200
</code></pre>


                
                
<div class="entry-shang text-center">
    
	    <p>「真诚赞赏，手留余香」</p>
	
	<button class="zs show-zs btn btn-bred">赞赏支持</button>
</div>
<div class="zs-modal-bg"></div>
<div class="zs-modal-box">
	<div class="zs-modal-head">
		<button type="button" class="close">×</button>
		<span class="author"><a href="https://masxhub.gitee.io/blog"><img src="/blog/img/favicon.png" />Mashaoxun Blog</a></span>
        
	        <p class="tip"><i></i><span>真诚赞赏，手留余香</span></p>
		
 
	</div>
	<div class="zs-modal-body">
		<div class="zs-modal-btns">
			<button class="btn btn-blink" data-num="2">2元</button>
			<button class="btn btn-blink" data-num="5">5元</button>
			<button class="btn btn-blink" data-num="10">10元</button>
			<button class="btn btn-blink" data-num="20">20元</button>
			<button class="btn btn-blink" data-num="50">50元</button>
			<button class="btn btn-blink" data-num="100">100元</button>
		</div>
		<div class="zs-modal-pay">
			<button class="btn btn-bred" id="pay-text">2元</button>
			<p>使用<span id="pay-type">微信</span>扫描二维码完成支付</p>
			<img src="/blog/img/reward/zanshang.jpg"  id="pay-image"/>
		</div>
	</div>
	<div class="zs-modal-footer">
		<label><input type="radio" name="zs-type" value="wechat" class="zs-type" checked="checked"><span ><span class="zs-wechat"><img src="/blog/img/reward/wechat-btn.png"/></span></label>
		<label><input type="radio" name="zs-type" value="alipay" class="zs-type" class="zs-alipay"><img src="/blog/img/reward/alipay-btn.png"/></span></label>
	</div>
</div>
<script type="text/javascript" src="/blog/js/reward.js"></script>

                

                <hr>
                <ul class="pager">
                    
                    <li class="previous">
                        <a href="/blog/post/2019-03-12-k8s-yaml-create/" data-toggle="tooltip" data-placement="top" title="kubectl生成yaml文件以及查看pod的yaml">&larr;
                            Previous Post</a>
                    </li>
                    
                    
                    <li class="next">
                        <a href="2019/04/30/swarm-profile/" data-toggle="tooltip" data-placement="top" title="docker swarm 学习一">Next
                            Post &rarr;</a>
                    </li>
                    
                </ul>

                
<div id="disqus-comment"></div>



            </div>
            
            <div class="
                col-lg-8 col-lg-offset-2
                col-md-10 col-md-offset-1
                sidebar-container">

                
                
                <section>
                    <hr class="hidden-sm hidden-xs">
                    <h5><a href="/tags/">FEATURED TAGS</a></h5>
                    <div class="tags">
                        
                        
                        
                        <a href="/blog/tags/ceph" title="ceph">
                            ceph
                        </a>
                        
                        
                        
                        
                        
                        <a href="/blog/tags/docker" title="docker">
                            docker
                        </a>
                        
                        
                        
                        <a href="/blog/tags/k8s" title="k8s">
                            k8s
                        </a>
                        
                        
                        
                        
                        
                        
                        
                        
                        
                        <a href="/blog/tags/%E5%AD%98%E5%82%A8" title="存储">
                            存储
                        </a>
                        
                        
                        
                        <a href="/blog/tags/%E7%9B%91%E6%8E%A7" title="监控">
                            监控
                        </a>
                        
                        
                        
                        
                    </div>
                </section>
                

                
                
                <section>
                    <hr>
                    <h5>FRIENDS</h5>
                    <ul class="list-inline">
                        
                        <li><a target="_blank" href="http://www.zrq.org.cn">瑞卿博客</a></li>
                        
                    </ul>
                </section>
                
            </div>
        </div>
    </div>
</article>




<footer>
    <div class="container">
        <div class="row">
            <div class="col-lg-8 col-lg-offset-2 col-md-10 col-md-offset-1">
                <ul class="list-inline text-center">
                   
                    
                    <li>
                        <a href="mailto:youremail@gmail.com">
                            <span class="fa-stack fa-lg">
                                <i class="fa fa-circle fa-stack-2x"></i>
                                <i class="fa fa-envelope fa-stack-1x fa-inverse"></i>
                            </span>
                        </a>
                    </li>
		    
                    
                    
                    
                    

                    

		    
                    
                    <li>
                        <a target="_blank" href="/blog/img/404.jpeg">
                            <span class="fa-stack fa-lg">
                                <i class="fa fa-circle fa-stack-2x"></i>
                                <i class="fa fa-wechat fa-stack-1x fa-inverse"></i>
                            </span>
                        </a>
                    </li>
		    
                    
                    <li>
                        <a target="_blank" href="https://github.com/yourgithub">
                            <span class="fa-stack fa-lg">
                                <i class="fa fa-circle fa-stack-2x"></i>
                                <i class="fa fa-github fa-stack-1x fa-inverse"></i>
                            </span>
                        </a>
                    </li>
		    
                    
                    
                </ul>
		<p class="copyright text-muted">
                    Copyright &copy; Mashaoxun Blog , 2019
                    <br>
                    <a href="https://themes.gohugo.io/hugo-theme-cleanwhite">CleanWhite Hugo Theme</a> by <a href="https://zhaohuabing.com">Huabing</a> |
                    <iframe
                        style="margin-left: 2px; margin-bottom:-5px;"
                        frameborder="0" scrolling="0" width="100px" height="20px"
                        src="https://ghbtns.com/github-btn.html?user=zhaohuabing&repo=hugo-theme-cleanwhite&type=star&count=true" >
                    </iframe>
                </p>
            </div>
        </div>
    </div>
</footer>




<script>
    function async(u, c) {
      var d = document, t = 'script',
          o = d.createElement(t),
          s = d.getElementsByTagName(t)[0];
      o.src = u;
      if (c) { o.addEventListener('load', function (e) { c(null, e); }, false); }
      s.parentNode.insertBefore(o, s);
    }
</script>






<script>
    
    if($('#tag_cloud').length !== 0){
        async("/js/jquery.tagcloud.js",function(){
            $.fn.tagcloud.defaults = {
                
                color: {start: '#bbbbee', end: '#0085a1'},
            };
            $('#tag_cloud a').tagcloud();
        })
    }
</script>


<script>
    async("https://cdnjs.cloudflare.com/ajax/libs/fastclick/1.0.6/fastclick.js", function(){
        var $nav = document.querySelector("nav");
        if($nav) FastClick.attach($nav);
    })
</script>


<script>
    (function(){
        var bp = document.createElement('script');
        var curProtocol = window.location.protocol.split(':')[0];
        if (curProtocol === 'https'){
       bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
      }
      else{
      bp.src = 'http://push.zhanzhang.baidu.com/push.js';
      }
        var s = document.getElementsByTagName("script")[0];
        s.parentNode.insertBefore(bp, s);
    })();
</script>







</body>
</html>
